Chat Data Analysis (WhatsApp Group)¶
Data Collection¶
Data collected by exporting chat without media (format txt)
Getting ready¶
Let us start setting up our enviroment so that we need to install required packages, libraries and so forth.
In [1]:
!pip install emoji
In [2]:
!git clone https://github.com/amueller/word_cloud.git
In [3]:
!pip install wordcloud
In [4]:
!pip install regex
In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import regex
import re
import emoji
import plotly.graph_objects as go
from collections import Counter
from os import path
from PIL import Image
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
%matplotlib inline
Preparing Data¶
Firstly, we need to parse and tokenize plain text (txt) in order to deploy infromation in meaningful way.
four tokens have been indentified: date, time, author and message: {Date}{Time}{Author}{Message}
Example: {01/01/2020}{9:00}{Javier}{Buenos días chicos, ¿como estáis?} Therefore, parsing and tokenization functions have been developed as follows:
In [2]:
def startDateTime(s):
pattern = '^([0-9]+)(\/)([0-9]+)(\/)([0-9]+) ([0-9]+):([0-9]+)[ ]?-'
result = re.match(pattern,s)
if result:
return True
return False
In [3]:
startDateTime('17/3/19 15:06 - Gabo: Hola')
Out[3]:
In [4]:
def findAuthor(s):
s = s.split(":")
if len(s) == 2:
return True
else:
return False
In [5]:
s = '17/3/19 15:06 - Gabo: Hola'
findAuthor(s)
Out[5]:
In [6]:
def dataPoint(line):
splitLine = line.split(' - ')
dateTime = splitLine[0]
date, time = dateTime.split(' ')
message = ' '.join(splitLine[1:])
if findAuthor(message):
splitMessage = message.split(': ')
author = splitMessage[0]
message = ' '.join(splitMessage[1:])
else:
author = None
return date, time, author, message
In [7]:
parsedData = []
cnvPath = 'Alicante.txt'
with open(cnvPath, encoding='utf-8') as fp:
fp.readline()
messageBuffer = []
date, time, autor = None, None, None
while True:
line = fp.readline()
if not line:
break
line = line.strip()
if startDateTime(line):
if len(messageBuffer) > 0:
parsedData.append([date, time, author,' '.join(messageBuffer)])
messageBuffer.clear()
date, time, author, message = dataPoint(line)
messageBuffer.append(message)
else:
messageBuffer.append(line)
In [8]:
df = pd.DataFrame(parsedData, columns = ['Date','Time','Author','Message'])
df["Date"] = pd.to_datetime(df["Date"])
In [9]:
df.head(5)
Out[9]:
In [10]:
df.info()
Data Visualization¶
Now, participating view is asked so that realize author's activity for each.
In [11]:
df.Author.unique()
Out[11]:
None: refers those messages that do not have authors, such as: "Has sido añadido al grupo" or links added in the conversation
Removing None's messages
In [12]:
df = df.dropna()
df.info()
In [13]:
df.Author.unique()
Out[13]:
Meaningful Stats¶
In [14]:
Total_msg = df.shape[0]
print(Total_msg)
In [15]:
Media_msg = df[df['Message']== '<Multimedia omitido>'].shape[0]
print(Media_msg)
In [16]:
def split_count(text):
emoji_lst = []
data = regex.findall(r'\X', text)
for word in data:
if any(char in emoji.UNICODE_EMOJI for char in word):
emoji_lst.append(word)
return emoji_lst
df["emoji"] = df["Message"].apply(split_count)
In [17]:
emojis = sum(df['emoji'].str.len())
print(emojis)
In [18]:
URLpattern = r'(https?://\S+)'
df['urlcount'] = df.Message.apply(lambda x: re.findall(URLpattern, x)).str.len()
In [19]:
links = np.sum(df.urlcount)
print(links)
In [20]:
print("Group Wise Stats")
print("Messages:",Total_msg)
print("Media:",Media_msg)
print("Emojis:",emojis)
print("Links:",links)
Message Processing¶
In [21]:
Media_msg_df = df[df['Message'] == '<Multimedia omitido>']
msg_df = df.drop(Media_msg_df.index)
msg_df.info()
Let's count the number of letters and words used by each author per message. To do so, 2 new columns have been added to the data frame as it follows:
In [22]:
msg_df['Letter_Count'] = msg_df['Message'].apply(lambda s : len(s))
msg_df['Word_Count'] = msg_df['Message'].apply(lambda s : len(s.split(' ')))
msg_df["MessageCount"]=1
In [23]:
msg_df.head(5)
Out[23]:
In [24]:
msg_df["emojicount"]= df['emoji'].str.len()
In [25]:
msg_df.head(5)
Out[25]:
Wise Stats by Author¶
In [26]:
l = msg_df.Author.unique()
for i in range(len(l)):
req_df = msg_df[msg_df['Author']== l[i]]
print(f'Stats of {l[i]} -')
print('Messages sent', req_df.shape[0])
words_per_msg = (np.sum(req_df['Word_Count']))/req_df.shape[0]
print('Words per message', words_per_msg)
media = Media_msg_df[Media_msg_df['Author'] == l[i]].shape[0]
print('Media Messages Sent', media)
emojis = sum(req_df['emoji'].str.len())
print('Emojis Sent', emojis)
links = sum(req_df["urlcount"])
print('Links Sent', links)
print()
Emoji Stats¶
Unique and most used emojis¶
Unique emojis used
In [27]:
total_emojis_list = list(set([a for b in msg_df.emoji for a in b]))
total_emojis = len(total_emojis_list)
print(total_emojis)
Most used emojis
In [28]:
total_emojis_list = list([a for b in msg_df.emoji for a in b])
emoji_dict = dict(Counter(total_emojis_list))
emoji_dict = sorted(emoji_dict.items(), key=lambda x: x[1], reverse=True)
print(emoji_dict)
In [29]:
emoji_df = pd.DataFrame(emoji_dict, columns=['emoji', 'count'])
emoji_df.head(11)
Out[29]:
Emoji distribution¶
In [30]:
import plotly.express as px
fig = px.pie(emoji_df, values='count', names='emoji')
fig.update_traces(textposition='inside', textinfo='percent+label')
fig.show()
WordCloud¶
In [31]:
text = " ".join(review for review in msg_df.Message)
print ("There are {} words in all the messages.".format(len(text)))
In [32]:
stopwords = set(STOPWORDS)
wordcloud = WordCloud(stopwords=stopwords, background_color="black").generate(text)
plt.figure( figsize=(12,6))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
By Author¶
In [33]:
l = msg_df.Author.unique()
for i in range(len(l)):
dummy_df = msg_df[msg_df['Author'] == l[i]]
text = " ".join(review for review in dummy_df.Message)
stopwords = set(STOPWORDS)
print('Author name',l[i])
wordcloud = WordCloud(stopwords=stopwords, background_color="black").generate(text)
plt.figure( figsize=(10,5))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
Day Distribution
In [34]:
def f(i):
l=["Monday", "Tuesday", "Wednesday","Thursday", "Friday", "Saturday","Sunday"]
return l[i];
day_df=pd.DataFrame(msg_df["Message"])
day_df['day_date'] = msg_df['Date'].dt.weekday
day_df['day_date'] = day_df['day_date'].apply(f)
day_df['MessageCount'] = 1
day = day_df.groupby("day_date").sum()
day.reset_index(inplace=True)
In [35]:
fig = px.line_polar(day, r='MessageCount', theta='day_date', line_close=True)
fig.update_traces(fill="toself")
fig.update_layout(
polar=dict(
radialaxis=dict(
visible=True,
)),
showlegend=False
)
fig.show()
In [122]:
date_df = msg_df.groupby("Date").sum()
date_df.reset_index(inplace=True)
fig = px.line(date_df, x="Date", y="MessageCount")
fig.update_xaxes(nticks=20)
fig.show()
In [123]:
date_df["rolling"] = date_df["MessageCount"].rolling(30).mean()
fig = px.line(date_df, x="Date", y="rolling")
fig.update_xaxes(nticks=20)
fig.show()
When are member most active?¶
Top 10
In [124]:
msg_df['Time'].value_counts().head(10).plot.barh()
plt.xlabel('Number of messages')
plt.ylabel('Time')
Out[124]:
Most active day¶
In [125]:
msg_df['Date'].value_counts().head(10).plot.barh()
print(msg_df['Date'].value_counts())
plt.xlabel('Number of Messages')
plt.ylabel('Date')
Out[125]:
In [ ]:
In [ ]: